{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:48:50.352373Z",
     "start_time": "2018-09-11T17:48:50.341906Z"
    }
   },
   "outputs": [],
   "source": [
    "# see http://icsdweb.aegean.gr/awid for more info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-12T01:23:43.095572Z",
     "start_time": "2018-09-12T01:23:43.070303Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:48:51.186105Z",
     "start_time": "2018-09-11T17:48:51.020660Z"
    },
    "code_folding": [
     0.0
    ]
   },
   "outputs": [],
   "source": [
    "# http://icsdweb.aegean.gr/awid/features.html\n",
    "\n",
    "# get the names of the features\n",
    "    \n",
    "features = ['frame.interface_id',\n",
    " 'frame.dlt',\n",
    " 'frame.offset_shift',\n",
    " 'frame.time_epoch',\n",
    " 'frame.time_delta',\n",
    " 'frame.time_delta_displayed',\n",
    " 'frame.time_relative',\n",
    " 'frame.len',\n",
    " 'frame.cap_len',\n",
    " 'frame.marked',\n",
    " 'frame.ignored',\n",
    " 'radiotap.version',\n",
    " 'radiotap.pad',\n",
    " 'radiotap.length',\n",
    " 'radiotap.present.tsft',\n",
    " 'radiotap.present.flags',\n",
    " 'radiotap.present.rate',\n",
    " 'radiotap.present.channel',\n",
    " 'radiotap.present.fhss',\n",
    " 'radiotap.present.dbm_antsignal',\n",
    " 'radiotap.present.dbm_antnoise',\n",
    " 'radiotap.present.lock_quality',\n",
    " 'radiotap.present.tx_attenuation',\n",
    " 'radiotap.present.db_tx_attenuation',\n",
    " 'radiotap.present.dbm_tx_power',\n",
    " 'radiotap.present.antenna',\n",
    " 'radiotap.present.db_antsignal',\n",
    " 'radiotap.present.db_antnoise',\n",
    " 'radiotap.present.rxflags',\n",
    " 'radiotap.present.xchannel',\n",
    " 'radiotap.present.mcs',\n",
    " 'radiotap.present.ampdu',\n",
    " 'radiotap.present.vht',\n",
    " 'radiotap.present.reserved',\n",
    " 'radiotap.present.rtap_ns',\n",
    " 'radiotap.present.vendor_ns',\n",
    " 'radiotap.present.ext',\n",
    " 'radiotap.mactime',\n",
    " 'radiotap.flags.cfp',\n",
    " 'radiotap.flags.preamble',\n",
    " 'radiotap.flags.wep',\n",
    " 'radiotap.flags.frag',\n",
    " 'radiotap.flags.fcs',\n",
    " 'radiotap.flags.datapad',\n",
    " 'radiotap.flags.badfcs',\n",
    " 'radiotap.flags.shortgi',\n",
    " 'radiotap.datarate',\n",
    " 'radiotap.channel.freq',\n",
    " 'radiotap.channel.type.turbo',\n",
    " 'radiotap.channel.type.cck',\n",
    " 'radiotap.channel.type.ofdm',\n",
    " 'radiotap.channel.type.2ghz',\n",
    " 'radiotap.channel.type.5ghz',\n",
    " 'radiotap.channel.type.passive',\n",
    " 'radiotap.channel.type.dynamic',\n",
    " 'radiotap.channel.type.gfsk',\n",
    " 'radiotap.channel.type.gsm',\n",
    " 'radiotap.channel.type.sturbo',\n",
    " 'radiotap.channel.type.half',\n",
    " 'radiotap.channel.type.quarter',\n",
    " 'radiotap.dbm_antsignal',\n",
    " 'radiotap.antenna',\n",
    " 'radiotap.rxflags.badplcp',\n",
    " 'wlan.fc.type_subtype',\n",
    " 'wlan.fc.version',\n",
    " 'wlan.fc.type',\n",
    " 'wlan.fc.subtype',\n",
    " 'wlan.fc.ds',\n",
    " 'wlan.fc.frag',\n",
    " 'wlan.fc.retry',\n",
    " 'wlan.fc.pwrmgt',\n",
    " 'wlan.fc.moredata',\n",
    " 'wlan.fc.protected',\n",
    " 'wlan.fc.order',\n",
    " 'wlan.duration',\n",
    " 'wlan.ra',\n",
    " 'wlan.da',\n",
    " 'wlan.ta',\n",
    " 'wlan.sa',\n",
    " 'wlan.bssid',\n",
    " 'wlan.frag',\n",
    " 'wlan.seq',\n",
    " 'wlan.bar.type',\n",
    " 'wlan.ba.control.ackpolicy',\n",
    " 'wlan.ba.control.multitid',\n",
    " 'wlan.ba.control.cbitmap',\n",
    " 'wlan.bar.compressed.tidinfo',\n",
    " 'wlan.ba.bm',\n",
    " 'wlan.fcs_good',\n",
    " 'wlan_mgt.fixed.capabilities.ess',\n",
    " 'wlan_mgt.fixed.capabilities.ibss',\n",
    " 'wlan_mgt.fixed.capabilities.cfpoll.ap',\n",
    " 'wlan_mgt.fixed.capabilities.privacy',\n",
    " 'wlan_mgt.fixed.capabilities.preamble',\n",
    " 'wlan_mgt.fixed.capabilities.pbcc',\n",
    " 'wlan_mgt.fixed.capabilities.agility',\n",
    " 'wlan_mgt.fixed.capabilities.spec_man',\n",
    " 'wlan_mgt.fixed.capabilities.short_slot_time',\n",
    " 'wlan_mgt.fixed.capabilities.apsd',\n",
    " 'wlan_mgt.fixed.capabilities.radio_measurement',\n",
    " 'wlan_mgt.fixed.capabilities.dsss_ofdm',\n",
    " 'wlan_mgt.fixed.capabilities.del_blk_ack',\n",
    " 'wlan_mgt.fixed.capabilities.imm_blk_ack',\n",
    " 'wlan_mgt.fixed.listen_ival',\n",
    " 'wlan_mgt.fixed.current_ap',\n",
    " 'wlan_mgt.fixed.status_code',\n",
    " 'wlan_mgt.fixed.timestamp',\n",
    " 'wlan_mgt.fixed.beacon',\n",
    " 'wlan_mgt.fixed.aid',\n",
    " 'wlan_mgt.fixed.reason_code',\n",
    " 'wlan_mgt.fixed.auth.alg',\n",
    " 'wlan_mgt.fixed.auth_seq',\n",
    " 'wlan_mgt.fixed.category_code',\n",
    " 'wlan_mgt.fixed.htact',\n",
    " 'wlan_mgt.fixed.chanwidth',\n",
    " 'wlan_mgt.fixed.fragment',\n",
    " 'wlan_mgt.fixed.sequence',\n",
    " 'wlan_mgt.tagged.all',\n",
    " 'wlan_mgt.ssid',\n",
    " 'wlan_mgt.ds.current_channel',\n",
    " 'wlan_mgt.tim.dtim_count',\n",
    " 'wlan_mgt.tim.dtim_period',\n",
    " 'wlan_mgt.tim.bmapctl.multicast',\n",
    " 'wlan_mgt.tim.bmapctl.offset',\n",
    " 'wlan_mgt.country_info.environment',\n",
    " 'wlan_mgt.rsn.version',\n",
    " 'wlan_mgt.rsn.gcs.type',\n",
    " 'wlan_mgt.rsn.pcs.count',\n",
    " 'wlan_mgt.rsn.akms.count',\n",
    " 'wlan_mgt.rsn.akms.type',\n",
    " 'wlan_mgt.rsn.capabilities.preauth',\n",
    " 'wlan_mgt.rsn.capabilities.no_pairwise',\n",
    " 'wlan_mgt.rsn.capabilities.ptksa_replay_counter',\n",
    " 'wlan_mgt.rsn.capabilities.gtksa_replay_counter',\n",
    " 'wlan_mgt.rsn.capabilities.mfpr',\n",
    " 'wlan_mgt.rsn.capabilities.mfpc',\n",
    " 'wlan_mgt.rsn.capabilities.peerkey',\n",
    " 'wlan_mgt.tcprep.trsmt_pow',\n",
    " 'wlan_mgt.tcprep.link_mrg',\n",
    " 'wlan.wep.iv',\n",
    " 'wlan.wep.key',\n",
    " 'wlan.wep.icv',\n",
    " 'wlan.tkip.extiv',\n",
    " 'wlan.ccmp.extiv',\n",
    " 'wlan.qos.tid',\n",
    " 'wlan.qos.priority',\n",
    " 'wlan.qos.eosp',\n",
    " 'wlan.qos.ack',\n",
    " 'wlan.qos.amsdupresent',\n",
    " 'wlan.qos.buf_state_indicated',\n",
    " 'wlan.qos.bit4',\n",
    " 'wlan.qos.txop_dur_req',\n",
    " 'wlan.qos.buf_state_indicated',\n",
    " 'data.len',\n",
    " 'class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:49:31.230772Z",
     "start_time": "2018-09-11T17:48:51.191479Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sinanozdemir/Desktop/personal_code/cyber/env/lib/python2.7/site-packages/pandas/io/parsers.py:678: UserWarning: Duplicate names specified. This will raise an error in the future.\n",
      "  return _read(filepath_or_buffer, kwds)\n",
      "/Users/sinanozdemir/Desktop/personal_code/cyber/env/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2714: DtypeWarning: Columns (37,38,39,40,41,42,43,44,45,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,74,88) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1795575, 155)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import a training set\n",
    "awid = pd.read_csv(\"../data/AWID-CLS-R-Trn.csv\", header=None, names=features)\n",
    "\n",
    "# see the number of rows/columns\n",
    "awid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:49:31.305996Z",
     "start_time": "2018-09-11T17:49:31.241224Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frame.interface_id</th>\n",
       "      <th>frame.dlt</th>\n",
       "      <th>frame.offset_shift</th>\n",
       "      <th>frame.time_epoch</th>\n",
       "      <th>frame.time_delta</th>\n",
       "      <th>frame.time_delta_displayed</th>\n",
       "      <th>frame.time_relative</th>\n",
       "      <th>frame.len</th>\n",
       "      <th>frame.cap_len</th>\n",
       "      <th>frame.marked</th>\n",
       "      <th>...</th>\n",
       "      <th>wlan.qos.priority</th>\n",
       "      <th>wlan.qos.eosp</th>\n",
       "      <th>wlan.qos.ack</th>\n",
       "      <th>wlan.qos.amsdupresent</th>\n",
       "      <th>wlan.qos.buf_state_indicated</th>\n",
       "      <th>wlan.qos.bit4</th>\n",
       "      <th>wlan.qos.txop_dur_req</th>\n",
       "      <th>wlan.qos.buf_state_indicated.1</th>\n",
       "      <th>data.len</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>261</td>\n",
       "      <td>261</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.024271</td>\n",
       "      <td>0.024271</td>\n",
       "      <td>0.024271</td>\n",
       "      <td>185</td>\n",
       "      <td>185</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.001631</td>\n",
       "      <td>0.001631</td>\n",
       "      <td>0.025902</td>\n",
       "      <td>185</td>\n",
       "      <td>185</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.055325</td>\n",
       "      <td>0.055325</td>\n",
       "      <td>0.081227</td>\n",
       "      <td>159</td>\n",
       "      <td>159</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.000415</td>\n",
       "      <td>0.000415</td>\n",
       "      <td>0.081642</td>\n",
       "      <td>54</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 155 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   frame.interface_id frame.dlt  frame.offset_shift  frame.time_epoch  \\\n",
       "0                   0         ?                 0.0      1.393661e+09   \n",
       "1                   0         ?                 0.0      1.393661e+09   \n",
       "2                   0         ?                 0.0      1.393661e+09   \n",
       "3                   0         ?                 0.0      1.393661e+09   \n",
       "4                   0         ?                 0.0      1.393661e+09   \n",
       "\n",
       "   frame.time_delta  frame.time_delta_displayed  frame.time_relative  \\\n",
       "0          0.000000                    0.000000             0.000000   \n",
       "1          0.024271                    0.024271             0.024271   \n",
       "2          0.001631                    0.001631             0.025902   \n",
       "3          0.055325                    0.055325             0.081227   \n",
       "4          0.000415                    0.000415             0.081642   \n",
       "\n",
       "   frame.len  frame.cap_len  frame.marked   ...    wlan.qos.priority  \\\n",
       "0        261            261             0   ...                    ?   \n",
       "1        185            185             0   ...                    ?   \n",
       "2        185            185             0   ...                    ?   \n",
       "3        159            159             0   ...                    ?   \n",
       "4         54             54             0   ...                    ?   \n",
       "\n",
       "   wlan.qos.eosp  wlan.qos.ack  wlan.qos.amsdupresent  \\\n",
       "0              ?             ?                      ?   \n",
       "1              ?             ?                      ?   \n",
       "2              ?             ?                      ?   \n",
       "3              ?             ?                      ?   \n",
       "4              ?             ?                      ?   \n",
       "\n",
       "   wlan.qos.buf_state_indicated  wlan.qos.bit4  wlan.qos.txop_dur_req  \\\n",
       "0                             ?              ?                      ?   \n",
       "1                             ?              ?                      ?   \n",
       "2                             ?              ?                      ?   \n",
       "3                             ?              ?                      ?   \n",
       "4                             ?              ?                      ?   \n",
       "\n",
       "   wlan.qos.buf_state_indicated.1  data.len   class  \n",
       "0                               ?         ?  normal  \n",
       "1                               ?         ?  normal  \n",
       "2                               ?         ?  normal  \n",
       "3                               ?         ?  normal  \n",
       "4                               ?         ?  normal  \n",
       "\n",
       "[5 rows x 155 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# they use ? as a null attribute. We will eventually have to replace them with None values\n",
    "awid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:49:31.601124Z",
     "start_time": "2018-09-11T17:49:31.316370Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "normal           0.909564\n",
       "injection        0.036411\n",
       "impersonation    0.027023\n",
       "flooding         0.027002\n",
       "Name: class, dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see the distribution of response vars\n",
    "awid['class'].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:49:44.941489Z",
     "start_time": "2018-09-11T17:49:31.607027Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "frame.interface_id                                0\n",
       "frame.dlt                                         0\n",
       "frame.offset_shift                                0\n",
       "frame.time_epoch                                  0\n",
       "frame.time_delta                                  0\n",
       "frame.time_delta_displayed                        0\n",
       "frame.time_relative                               0\n",
       "frame.len                                         0\n",
       "frame.cap_len                                     0\n",
       "frame.marked                                      0\n",
       "frame.ignored                                     0\n",
       "radiotap.version                                  0\n",
       "radiotap.pad                                      0\n",
       "radiotap.length                                   0\n",
       "radiotap.present.tsft                             0\n",
       "radiotap.present.flags                            0\n",
       "radiotap.present.rate                             0\n",
       "radiotap.present.channel                          0\n",
       "radiotap.present.fhss                             0\n",
       "radiotap.present.dbm_antsignal                    0\n",
       "radiotap.present.dbm_antnoise                     0\n",
       "radiotap.present.lock_quality                     0\n",
       "radiotap.present.tx_attenuation                   0\n",
       "radiotap.present.db_tx_attenuation                0\n",
       "radiotap.present.dbm_tx_power                     0\n",
       "radiotap.present.antenna                          0\n",
       "radiotap.present.db_antsignal                     0\n",
       "radiotap.present.db_antnoise                      0\n",
       "radiotap.present.rxflags                          0\n",
       "radiotap.present.xchannel                         0\n",
       "                                                 ..\n",
       "wlan_mgt.rsn.version                              0\n",
       "wlan_mgt.rsn.gcs.type                             0\n",
       "wlan_mgt.rsn.pcs.count                            0\n",
       "wlan_mgt.rsn.akms.count                           0\n",
       "wlan_mgt.rsn.akms.type                            0\n",
       "wlan_mgt.rsn.capabilities.preauth                 0\n",
       "wlan_mgt.rsn.capabilities.no_pairwise             0\n",
       "wlan_mgt.rsn.capabilities.ptksa_replay_counter    0\n",
       "wlan_mgt.rsn.capabilities.gtksa_replay_counter    0\n",
       "wlan_mgt.rsn.capabilities.mfpr                    0\n",
       "wlan_mgt.rsn.capabilities.mfpc                    0\n",
       "wlan_mgt.rsn.capabilities.peerkey                 0\n",
       "wlan_mgt.tcprep.trsmt_pow                         0\n",
       "wlan_mgt.tcprep.link_mrg                          0\n",
       "wlan.wep.iv                                       0\n",
       "wlan.wep.key                                      0\n",
       "wlan.wep.icv                                      0\n",
       "wlan.tkip.extiv                                   0\n",
       "wlan.ccmp.extiv                                   0\n",
       "wlan.qos.tid                                      0\n",
       "wlan.qos.priority                                 0\n",
       "wlan.qos.eosp                                     0\n",
       "wlan.qos.ack                                      0\n",
       "wlan.qos.amsdupresent                             0\n",
       "wlan.qos.buf_state_indicated                      0\n",
       "wlan.qos.bit4                                     0\n",
       "wlan.qos.txop_dur_req                             0\n",
       "wlan.qos.buf_state_indicated.1                    0\n",
       "data.len                                          0\n",
       "class                                             0\n",
       "Length: 155, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# claims there are no null values because of the ?'s'\n",
    "awid.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:50:21.202504Z",
     "start_time": "2018-09-11T17:49:44.948005Z"
    }
   },
   "outputs": [],
   "source": [
    "# replace the ? marks with None\n",
    "awid.replace({\"?\": None}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:50:31.946653Z",
     "start_time": "2018-09-11T17:50:21.212174Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "frame.interface_id                                      0\n",
       "frame.dlt                                         1795575\n",
       "frame.offset_shift                                      0\n",
       "frame.time_epoch                                        0\n",
       "frame.time_delta                                        0\n",
       "frame.time_delta_displayed                              0\n",
       "frame.time_relative                                     0\n",
       "frame.len                                               0\n",
       "frame.cap_len                                           0\n",
       "frame.marked                                            0\n",
       "frame.ignored                                           0\n",
       "radiotap.version                                        0\n",
       "radiotap.pad                                            0\n",
       "radiotap.length                                         0\n",
       "radiotap.present.tsft                                   0\n",
       "radiotap.present.flags                                  0\n",
       "radiotap.present.rate                                   0\n",
       "radiotap.present.channel                                0\n",
       "radiotap.present.fhss                                   0\n",
       "radiotap.present.dbm_antsignal                          0\n",
       "radiotap.present.dbm_antnoise                           0\n",
       "radiotap.present.lock_quality                           0\n",
       "radiotap.present.tx_attenuation                         0\n",
       "radiotap.present.db_tx_attenuation                      0\n",
       "radiotap.present.dbm_tx_power                           0\n",
       "radiotap.present.antenna                                0\n",
       "radiotap.present.db_antsignal                           0\n",
       "radiotap.present.db_antnoise                            0\n",
       "radiotap.present.rxflags                                0\n",
       "radiotap.present.xchannel                               0\n",
       "                                                   ...   \n",
       "wlan_mgt.rsn.version                              1718631\n",
       "wlan_mgt.rsn.gcs.type                             1718631\n",
       "wlan_mgt.rsn.pcs.count                            1718631\n",
       "wlan_mgt.rsn.akms.count                           1718633\n",
       "wlan_mgt.rsn.akms.type                            1718651\n",
       "wlan_mgt.rsn.capabilities.preauth                 1718633\n",
       "wlan_mgt.rsn.capabilities.no_pairwise             1718633\n",
       "wlan_mgt.rsn.capabilities.ptksa_replay_counter    1718633\n",
       "wlan_mgt.rsn.capabilities.gtksa_replay_counter    1718633\n",
       "wlan_mgt.rsn.capabilities.mfpr                    1718633\n",
       "wlan_mgt.rsn.capabilities.mfpc                    1718633\n",
       "wlan_mgt.rsn.capabilities.peerkey                 1718633\n",
       "wlan_mgt.tcprep.trsmt_pow                         1795536\n",
       "wlan_mgt.tcprep.link_mrg                          1795536\n",
       "wlan.wep.iv                                        944820\n",
       "wlan.wep.key                                       909831\n",
       "wlan.wep.icv                                       944820\n",
       "wlan.tkip.extiv                                   1763655\n",
       "wlan.ccmp.extiv                                   1792506\n",
       "wlan.qos.tid                                      1133234\n",
       "wlan.qos.priority                                 1133234\n",
       "wlan.qos.eosp                                     1279874\n",
       "wlan.qos.ack                                      1133234\n",
       "wlan.qos.amsdupresent                             1134226\n",
       "wlan.qos.buf_state_indicated                      1795575\n",
       "wlan.qos.bit4                                     1648935\n",
       "wlan.qos.txop_dur_req                             1648935\n",
       "wlan.qos.buf_state_indicated.1                    1279874\n",
       "data.len                                           903021\n",
       "class                                                   0\n",
       "Length: 155, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Many missing pieces of data!\n",
    "awid.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:50:42.751148Z",
     "start_time": "2018-09-11T17:50:31.950979Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(72,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# goal is to remove columns that have over 50% of missing data\n",
    "columns_with_mostly_null_data = awid.columns[awid.isnull().mean() >= 0.5]\n",
    "\n",
    "# 72 columns are going to be affected!\n",
    "columns_with_mostly_null_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:50:44.330318Z",
     "start_time": "2018-09-11T17:50:42.756922Z"
    }
   },
   "outputs": [],
   "source": [
    "# drop the columns with over half missing data\n",
    "awid.drop(columns_with_mostly_null_data, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:50:44.359876Z",
     "start_time": "2018-09-11T17:50:44.340246Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1795575, 83)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "awid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:50:50.514912Z",
     "start_time": "2018-09-11T17:50:44.367547Z"
    }
   },
   "outputs": [],
   "source": [
    "# now drop the rows that have missing values\n",
    "awid.dropna(inplace=True)  # drop rows with null data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-12T01:26:00.631348Z",
     "start_time": "2018-09-12T01:26:00.616544Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1339406, 83)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lost 456,169 rows\n",
    "awid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:50:50.722700Z",
     "start_time": "2018-09-11T17:50:50.520709Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "normal           0.878763\n",
       "injection        0.048812\n",
       "impersonation    0.036227\n",
       "flooding         0.036198\n",
       "Name: class, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# doesn't affect our distribution too much\n",
    "\n",
    "# 0.878763  is our null accuracy. Our model must be better than this number to be a contender\n",
    "awid['class'].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:21:26.505275Z",
     "start_time": "2018-09-11T18:21:25.763378Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1339406, 45)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# only select numerical columns for our ML algorithms, there should be more..\n",
    "awid.select_dtypes(['number']).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:22:52.129077Z",
     "start_time": "2018-09-11T18:22:21.440765Z"
    }
   },
   "outputs": [],
   "source": [
    "# transform all columns into numerical dtypes\n",
    "for col in awid.columns:\n",
    "    awid[col] = pd.to_numeric(awid[col], errors='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:22:55.225156Z",
     "start_time": "2018-09-11T18:22:52.132862Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1339406, 74)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# that makes more sense\n",
    "awid.select_dtypes(['number']).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-12T01:27:21.214774Z",
     "start_time": "2018-09-12T01:27:10.871973Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frame.interface_id</th>\n",
       "      <th>frame.offset_shift</th>\n",
       "      <th>frame.time_epoch</th>\n",
       "      <th>frame.time_delta</th>\n",
       "      <th>frame.time_delta_displayed</th>\n",
       "      <th>frame.time_relative</th>\n",
       "      <th>frame.len</th>\n",
       "      <th>frame.cap_len</th>\n",
       "      <th>frame.marked</th>\n",
       "      <th>frame.ignored</th>\n",
       "      <th>...</th>\n",
       "      <th>wlan.fc.frag</th>\n",
       "      <th>wlan.fc.retry</th>\n",
       "      <th>wlan.fc.pwrmgt</th>\n",
       "      <th>wlan.fc.moredata</th>\n",
       "      <th>wlan.fc.protected</th>\n",
       "      <th>wlan.fc.order</th>\n",
       "      <th>wlan.duration</th>\n",
       "      <th>wlan.frag</th>\n",
       "      <th>wlan.seq</th>\n",
       "      <th>wlan.fcs_good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1339406.0</td>\n",
       "      <td>1339406.0</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1339406.0</td>\n",
       "      <td>1339406.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1339406.0</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1.339406e+06</td>\n",
       "      <td>1339406.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393663e+09</td>\n",
       "      <td>2.435200e-03</td>\n",
       "      <td>2.435200e-03</td>\n",
       "      <td>1.842787e+03</td>\n",
       "      <td>6.771728e+02</td>\n",
       "      <td>6.771728e+02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5.024615e-04</td>\n",
       "      <td>1.703964e-01</td>\n",
       "      <td>2.870825e-02</td>\n",
       "      <td>2.376427e-03</td>\n",
       "      <td>6.612961e-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.028715e+02</td>\n",
       "      <td>2.165885e-03</td>\n",
       "      <td>1.910773e+03</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.370012e+02</td>\n",
       "      <td>6.827733e-03</td>\n",
       "      <td>6.827733e-03</td>\n",
       "      <td>8.370012e+02</td>\n",
       "      <td>6.997003e+02</td>\n",
       "      <td>6.997003e+02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.241003e-02</td>\n",
       "      <td>3.759808e-01</td>\n",
       "      <td>1.669853e-01</td>\n",
       "      <td>4.869067e-02</td>\n",
       "      <td>4.732692e-01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.237048e+02</td>\n",
       "      <td>1.176732e-01</td>\n",
       "      <td>1.200198e+03</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393661e+09</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.400000e+01</td>\n",
       "      <td>5.400000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393662e+09</td>\n",
       "      <td>4.430000e-04</td>\n",
       "      <td>4.430000e-04</td>\n",
       "      <td>1.103988e+03</td>\n",
       "      <td>9.800000e+01</td>\n",
       "      <td>9.800000e+01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.400000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>8.410000e+02</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393663e+09</td>\n",
       "      <td>7.790000e-04</td>\n",
       "      <td>7.790000e-04</td>\n",
       "      <td>1.894319e+03</td>\n",
       "      <td>1.590000e+02</td>\n",
       "      <td>1.590000e+02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.400000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.812000e+03</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393664e+09</td>\n",
       "      <td>1.393000e-03</td>\n",
       "      <td>1.393000e-03</td>\n",
       "      <td>2.374825e+03</td>\n",
       "      <td>1.552000e+03</td>\n",
       "      <td>1.552000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.580000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.941000e+03</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.393665e+09</td>\n",
       "      <td>3.502630e-01</td>\n",
       "      <td>3.502630e-01</td>\n",
       "      <td>3.599571e+03</td>\n",
       "      <td>1.570000e+03</td>\n",
       "      <td>1.570000e+03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.200000e+02</td>\n",
       "      <td>1.200000e+01</td>\n",
       "      <td>4.095000e+03</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 74 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       frame.interface_id  frame.offset_shift  frame.time_epoch  \\\n",
       "count           1339406.0           1339406.0      1.339406e+06   \n",
       "mean                  0.0                 0.0      1.393663e+09   \n",
       "std                   0.0                 0.0      8.370012e+02   \n",
       "min                   0.0                 0.0      1.393661e+09   \n",
       "25%                   0.0                 0.0      1.393662e+09   \n",
       "50%                   0.0                 0.0      1.393663e+09   \n",
       "75%                   0.0                 0.0      1.393664e+09   \n",
       "max                   0.0                 0.0      1.393665e+09   \n",
       "\n",
       "       frame.time_delta  frame.time_delta_displayed  frame.time_relative  \\\n",
       "count      1.339406e+06                1.339406e+06         1.339406e+06   \n",
       "mean       2.435200e-03                2.435200e-03         1.842787e+03   \n",
       "std        6.827733e-03                6.827733e-03         8.370012e+02   \n",
       "min        0.000000e+00                0.000000e+00         0.000000e+00   \n",
       "25%        4.430000e-04                4.430000e-04         1.103988e+03   \n",
       "50%        7.790000e-04                7.790000e-04         1.894319e+03   \n",
       "75%        1.393000e-03                1.393000e-03         2.374825e+03   \n",
       "max        3.502630e-01                3.502630e-01         3.599571e+03   \n",
       "\n",
       "          frame.len  frame.cap_len  frame.marked  frame.ignored  \\\n",
       "count  1.339406e+06   1.339406e+06     1339406.0      1339406.0   \n",
       "mean   6.771728e+02   6.771728e+02           0.0            0.0   \n",
       "std    6.997003e+02   6.997003e+02           0.0            0.0   \n",
       "min    5.400000e+01   5.400000e+01           0.0            0.0   \n",
       "25%    9.800000e+01   9.800000e+01           0.0            0.0   \n",
       "50%    1.590000e+02   1.590000e+02           0.0            0.0   \n",
       "75%    1.552000e+03   1.552000e+03           0.0            0.0   \n",
       "max    1.570000e+03   1.570000e+03           0.0            0.0   \n",
       "\n",
       "           ...        wlan.fc.frag  wlan.fc.retry  wlan.fc.pwrmgt  \\\n",
       "count      ...        1.339406e+06   1.339406e+06    1.339406e+06   \n",
       "mean       ...        5.024615e-04   1.703964e-01    2.870825e-02   \n",
       "std        ...        2.241003e-02   3.759808e-01    1.669853e-01   \n",
       "min        ...        0.000000e+00   0.000000e+00    0.000000e+00   \n",
       "25%        ...        0.000000e+00   0.000000e+00    0.000000e+00   \n",
       "50%        ...        0.000000e+00   0.000000e+00    0.000000e+00   \n",
       "75%        ...        0.000000e+00   0.000000e+00    0.000000e+00   \n",
       "max        ...        1.000000e+00   1.000000e+00    1.000000e+00   \n",
       "\n",
       "       wlan.fc.moredata  wlan.fc.protected  wlan.fc.order  wlan.duration  \\\n",
       "count      1.339406e+06       1.339406e+06      1339406.0   1.339406e+06   \n",
       "mean       2.376427e-03       6.612961e-01            0.0   1.028715e+02   \n",
       "std        4.869067e-02       4.732692e-01            0.0   1.237048e+02   \n",
       "min        0.000000e+00       0.000000e+00            0.0   0.000000e+00   \n",
       "25%        0.000000e+00       0.000000e+00            0.0   4.400000e+01   \n",
       "50%        0.000000e+00       1.000000e+00            0.0   4.400000e+01   \n",
       "75%        0.000000e+00       1.000000e+00            0.0   2.580000e+02   \n",
       "max        1.000000e+00       1.000000e+00            0.0   3.200000e+02   \n",
       "\n",
       "          wlan.frag      wlan.seq  wlan.fcs_good  \n",
       "count  1.339406e+06  1.339406e+06      1339406.0  \n",
       "mean   2.165885e-03  1.910773e+03            1.0  \n",
       "std    1.176732e-01  1.200198e+03            0.0  \n",
       "min    0.000000e+00  0.000000e+00            1.0  \n",
       "25%    0.000000e+00  8.410000e+02            1.0  \n",
       "50%    0.000000e+00  1.812000e+03            1.0  \n",
       "75%    0.000000e+00  2.941000e+03            1.0  \n",
       "max    1.200000e+01  4.095000e+03            1.0  \n",
       "\n",
       "[8 rows x 74 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# basic descroptive statistics\n",
    "awid.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-12T01:33:02.673468Z",
     "start_time": "2018-09-12T01:33:01.649555Z"
    }
   },
   "outputs": [],
   "source": [
    "X, y = awid.select_dtypes(['number']), awid['class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:44:05.911433Z",
     "start_time": "2018-09-11T18:43:57.554881Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB(priors=None)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# do a basic naive bayes fitting\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "nb = GaussianNB()\n",
    "\n",
    "# fit our model to the data\n",
    "nb.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:28:53.001626Z",
     "start_time": "2018-09-11T18:28:17.495406Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(389185, 83)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read in the test data and do the same transformations to it to match the training data\n",
    "awid_test = pd.read_csv(\"../data/AWID-CLS-R-Tst.csv\", header=None, names=features)\n",
    "# drop the problematic columns\n",
    "awid_test.drop(columns_with_mostly_null_data, axis=1, inplace=True)\n",
    "# replace ? with None\n",
    "awid_test.replace({\"?\": None}, inplace=True)\n",
    "# drop the rows with null data\n",
    "awid_test.dropna(inplace=True)  # drop rows with null data\n",
    "# convert columns to numerical values\n",
    "for col in awid_test.columns:\n",
    "    awid_test[col] = pd.to_numeric(awid_test[col], errors='ignore')\n",
    "awid_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T17:51:13.811466Z",
     "start_time": "2018-09-11T17:51:07.434162Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:44:09.361512Z",
     "start_time": "2018-09-11T18:44:09.353605Z"
    }
   },
   "outputs": [],
   "source": [
    "# basic metric, accuracy\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-12T01:33:08.115932Z",
     "start_time": "2018-09-12T01:33:04.739496Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.26535452291326744"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = awid_test.select_dtypes(['number'])\n",
    "y_test = awid_test['class']\n",
    "\n",
    "# simple function to test the accuracy of a model fitted on training data on our testing data\n",
    "def get_test_accuracy_of(model):\n",
    "    y_preds = model.predict(X_test)\n",
    "    return accuracy_score(y_preds, y_test)\n",
    "    \n",
    "# naive abyes does very poorly on its own!\n",
    "get_test_accuracy_of(nb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:44:36.553216Z",
     "start_time": "2018-09-11T18:44:12.841324Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.015773989233911892"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lr = LogisticRegression()\n",
    "\n",
    "lr.fit(X, y)\n",
    "\n",
    "# Logistic Regressions does even worse\n",
    "get_test_accuracy_of(lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:44:50.790091Z",
     "start_time": "2018-09-11T18:44:39.624541Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9336639387437851"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "tree = DecisionTreeClassifier()\n",
    "\n",
    "tree.fit(X, y)\n",
    "\n",
    "# Tree does very well!\n",
    "get_test_accuracy_of(tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:44:51.593791Z",
     "start_time": "2018-09-11T18:44:50.795143Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>frame.len</td>\n",
       "      <td>0.230466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>frame.time_delta</td>\n",
       "      <td>0.221151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>wlan.fc.protected</td>\n",
       "      <td>0.145760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>wlan.duration</td>\n",
       "      <td>0.127612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>frame.time_relative</td>\n",
       "      <td>0.079571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>frame.cap_len</td>\n",
       "      <td>0.059702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>wlan.fc.type</td>\n",
       "      <td>0.040192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>wlan.seq</td>\n",
       "      <td>0.026807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>wlan.fc.retry</td>\n",
       "      <td>0.019807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>radiotap.dbm_antsignal</td>\n",
       "      <td>0.014195</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   feature  importance\n",
       "6                frame.len    0.230466\n",
       "3         frame.time_delta    0.221151\n",
       "68       wlan.fc.protected    0.145760\n",
       "70           wlan.duration    0.127612\n",
       "5      frame.time_relative    0.079571\n",
       "7            frame.cap_len    0.059702\n",
       "62            wlan.fc.type    0.040192\n",
       "72                wlan.seq    0.026807\n",
       "65           wlan.fc.retry    0.019807\n",
       "58  radiotap.dbm_antsignal    0.014195"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see gini scores of decision tree features\n",
    "pd.DataFrame({'feature':awid.select_dtypes(['number']).columns, \n",
    "              'importance':tree.feature_importances_}).sort_values('importance', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:45:07.603342Z",
     "start_time": "2018-09-11T18:44:51.600315Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9297326464277914"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest = RandomForestClassifier()\n",
    "\n",
    "forest.fit(X, y)\n",
    "\n",
    "# Random Forest does slightly worse\n",
    "get_test_accuracy_of(forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:01:06.692457Z",
     "start_time": "2018-09-11T18:01:06.681432Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:48:34.877155Z",
     "start_time": "2018-09-11T18:45:17.876024Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9254930174595629"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a pipeline that will scale the numerical data and then feed the resulting data into a decision tree\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "preprocessing = Pipeline([\n",
    "    (\"scale\", StandardScaler()),\n",
    "])\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    (\"preprocessing\", preprocessing),\n",
    "    (\"classifier\", DecisionTreeClassifier())\n",
    "])\n",
    "\n",
    "# try varying levels of depth\n",
    "params = {\n",
    "    \"classifier__max_depth\": [None, 3, 5, 10], \n",
    "         }\n",
    "\n",
    "# instantiate a gridsearch module\n",
    "grid = GridSearchCV(pipeline, params)\n",
    "# fit the module\n",
    "grid.fit(X, y)\n",
    "\n",
    "# test the best model\n",
    "get_test_accuracy_of(grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-11T18:52:31.157867Z",
     "start_time": "2018-09-11T18:48:34.885991Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9348176317175636"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# try the same thing with a random forest\n",
    "\n",
    "preprocessing = Pipeline([\n",
    "    (\"scale\", StandardScaler()),\n",
    "])\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    (\"preprocessing\", preprocessing),\n",
    "    (\"classifier\", RandomForestClassifier())\n",
    "])\n",
    "\n",
    "# try varying levels of depth\n",
    "params = {\n",
    "    \"classifier__max_depth\": [None, 3, 5, 10], \n",
    "         }\n",
    "\n",
    "grid = GridSearchCV(pipeline, params)\n",
    "grid.fit(X, y)\n",
    "# best accuracy so far!\n",
    "get_test_accuracy_of(grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
